# FILE: WebDocument.py
# DESC: Represents a collection of WebEntity objects

from web.entity.WebEntity import WebEntity
from web.entity.PriceTag import PriceTag
from web.entity.Literal import Literal 
from geo.entity.GeoContext import GeoContext
from infra.logging import logger
from infra.lexical import LexicalAnalyser as lexanal

import re

MAX_SURROUNDING_TOKENS = 25
MAX_SURROUNDING_TEXT_LENGTH = 512

class WebDocument(WebEntity):

	def __init__(self, id, customer_id, abs_url, rel_url, 
                 nameEntity, images, whenDate, duration, 
                 priceTag, lastCrawl, contentLength):
		WebEntity.__init__(self, id, nameEntity.toString())
		self.customerId_ = customer_id
		self.abs_url_ = abs_url 
		self.rel_url_ = rel_url 
		self.nameEntity_ = nameEntity
		# REVISIT: image incarnation not yet supported
		# self.images_ = images 
		self.images_ = []
		self.whenDate_ = whenDate	
		self.duration_ = duration	
		self.priceTag_ = priceTag
		self.lastCrawl_ = lastCrawl 
		self.contentLength_ = contentLength
		self.webDocs_ = {}
		self.literals_ = [] 
		self.currentWhenDate_ = None
		self.currentDuration_ = None
		self.currentCurrency_ = None
		self.currentPriceTag_ = None
		self.geoContext_ = GeoContext()
		

		log = logger.getLogger()
		log.debug("%s = %s" % (self, self.__dict__))


	def getAbsoluteUrl(self):
		return self.abs_url_


	def getRelativeUrl(self):
		return self.rel_url_


	def setContentLength(self, contentLength):
		self.contentLength_ = contentLength


	def getContentLength(self):
		return self.contentLength_


	def getLastCrawl(self):
		return self.lastCrawl_


	def isCrawled(self):
		return self.lastCrawl_ is not None


	def isPersisted(self):
		return self.id_ != -1


	def isRoot(self):
		return self.rel_url_ == "/"


	def hasLiteralName(self):
		return isinstance(self.nameEntity_, Literal) and self.utf8Name_.lower().find("http") == -1


	def onWebDocument(self, webDoc):
		log = logger.getLogger()
		if not self.webDocs_.has_key(webDoc.getRelativeUrl()):
			self.webDocs_[webDoc.getRelativeUrl()] = webDoc
			# REVISIT: don't include referenced WebDocs to namedEntities
			# it leads to too many irrelevant links in GMarker infoWindow 
			#self.namedEntityList_.extend(webDoc.getNamedEntities())
			log.debug("Added webDocument = %s to dictionary." % webDoc.dump())
			
			if self.currentPriceTag_:
				webDoc.setPriceTag(self.currentPriceTag_)
				self.currentPriceTag_ = None

				if self.currentWhenDate_:
					webDoc.setWhenDate(self.currentWhenDate_)
					self.currentWhenDate_ = None

				if self.currentDuration_:
					webDoc.setDuration(self.currentDuration_)
					self.currentDuration_ = None

				# REVISIT: not sure how this works
				if not webDoc.hasLiteralName():
					names = []
					for literal in reversed(self.literals_):
						names.append(literal.getName())
						if literal.hasNamedEntities():
							webDoc.setName(" ".join(reversed(names)))
							break

			
	def onLiteral(self, literal):
		log = logger.getLogger()
		self.literals_.append(literal)
		literal.setId(len(self.literals_))
		log.debug("Added literal = %s to list." % literal.dump())
		if literal.hasNamedEntities():
			self.namedEntityList_.extend(literal.getNamedEntities())
			

	def setTextBefore(self, namedEntity):
		webEntity = namedEntity.getWebEntity()
		webEntityName = webEntity.getName()
		webEntityId = webEntity.getId()
		textBefore = webEntityName[0:webEntityName.find(namedEntity.getName())]
		numToksTot = textBefore.count(' ')
		if numToksTot > MAX_SURROUNDING_TOKENS:
			textBefore = ' '.join(textBefore.split(' ')[numToksTot - MAX_SURROUNDING_TOKENS: numToksTot])
		else:
			for literal in self.literals_[0:webEntityId]:
				numToks = literal.getNumTokens() 
				if numToksTot + numToks <= MAX_SURROUNDING_TOKENS:
					textBefore = lexanal.sentcat(literal.getName(), textBefore) + ' '
					numToksTot += numToks
				else:
					textBefore = lexanal.sentcat(' '.join(literal.getTokens()[numToks - MAX_SURROUNDING_TOKENS - numToksTot: numToks]) \
						, textBefore) 
					break
		namedEntity.setTextBefore(textBefore[0:MAX_SURROUNDING_TEXT_LENGTH])
					

	def setTextAfter(self, namedEntity):
		webEntity = namedEntity.getWebEntity()
		webEntityName = webEntity.getName()
		webEntityId = webEntity.getId()
		textAfter = webEntityName[webEntityName.find(namedEntity.getName()) + namedEntity.length():len(webEntityName)]
		
		numToksTot = textAfter.count(' ')
		if numToksTot > MAX_SURROUNDING_TOKENS:
			textAfter = ' '.join(textAfter.split(' ')[0:MAX_SURROUNDING_TOKENS])
		else:
			for literal in self.literals_[webEntityId:len(self.literals_)]:
				numToks = literal.getNumTokens() 
				if numToksTot + numToks <= MAX_SURROUNDING_TOKENS:
					textAfter = lexanal.sentcat(textAfter, literal.getName()) 
					numToksTot += numToks
				else:
					textAfter = lexanal.sentcat(textAfter, ' '.join(literal.getTokens()[0:MAX_SURROUNDING_TOKENS - numToksTot]))
					break
		namedEntity.setTextAfter(textAfter[0:MAX_SURROUNDING_TEXT_LENGTH])




	def onCurrency(self, currency):
		log = logger.getLogger()
		self.currentCurrency_ = currency
		log.debug("set currency to = %s" % currency.dump())



	def onImage(self, image):
		log = logger.getLogger()
		# REVISIT: once Image incarnation is implemented
		# replace list by map<image.getSource(), image>
		
		self.images_.append(image)
		log.debug("Added image = %s to list." % image.dump())


	def onDate(self, date):
		log = logger.getLogger()
		self.currentWhenDate_ = date
		log.debug("set date to = %s" % date.dump())


	def onDuration(self, duration):
		log = logger.getLogger()
		self.currentDuration_ = duration
		log.debug("set duration to = %s" % duration.dump())


	def onNumber(self, number):
		if self.currentCurrency_:
			log = logger.getLogger()
			self.currentPriceTag_ = PriceTag(self.currentCurrency_, number)
			log.debug("set priceTag to = %s" % self.currentPriceTag_.dump())
			self.currentCurrency_ = None


	def onPriceTag(self, priceTag):
		pass


	def setPriceTag(self, priceTag):
		self.priceTag_ = priceTag


	def getPriceTag(self):
		return self.priceTag_


	def setWhenDate(self, whenDate):
		self.whenDate_ = whenDate


	def getWhenDate(self):
		return self.whenDate_


	def setDuration(self, duration):
		self.duration_ = duration


	def getDuration(self):
		return self.duration_


	def setLastCrawl(self, date):
		self.lastCrawl_ = date

	
	def imagesToString(self, numImages = -1):
		i = numImages
		if i == -1 or i > len(self.images_):
			i = len(self.images_)
		return " ".join(["%s" % (image.getSource()) for image in self.images_[0:i]])


	def setCustomerId(self, customerId):
		self.customerId_ = customerId


	def getCustomerId(self):
		return self.customerId_


	def toInsertTuple(self, imageNumLimit = 5, nameLenLimit = 256):
		name = self.utf8Name_ 
		if len(self.utf8Name_) > nameLenLimit:
			name = self.utf8Name_[:nameLenLimit]

		return (self.customerId_, self.rel_url_, name, 
                self.imagesToString(imageNumLimit), self.whenDate_.toString(),
                self.duration_.toString(), self.priceTag_.toString(),
                self.lastCrawl_, self.contentLength_)


	def toUpdateTuple(self, imageNumLimit = 5, nameLenLimit = 256):
		name = self.utf8Name_ 
		if len(self.utf8Name_) > nameLenLimit:
			name = self.utf8Name_[:nameLenLimit]

		return (self.customerId_, self.rel_url_, name, 
                self.imagesToString(imageNumLimit), self.whenDate_.toString(),
                self.duration_.toString(), self.priceTag_.toString(),
                self.lastCrawl_, self.contentLength_, self.id_)


	def getKeyWords(self): 
		return self.keyWords_


	def dump(self):
		return self.__dict__


	def getNamedContext(self):
		return self.geoContext_


	def getTokens(self, predicate = None):
		tokens = []
		for literal in self.literals_:
			tokens.extend(literal.getTokens(predicate))
		for webDoc in self.webDocs_.itervalues():
			tokens.extend(webDoc.getTokens(predicate))
		return set(tokens)


	def getLiterals(self):
		return self.literals_


	def getWebDocuments(self):
		return self.webDocs_.itervalues()


	def hasWebDocument(self, geoName):
		rv = False
		nameList = geoName.getNames(altNames = True)
		for webDoc in self.webDocs_.itervalues():
			if webDoc.getName() and webDoc.getName() in nameList:
				rv = True
				break

		return rv
	
	
	def hasLiteral(self, geoName):
		rv = False
		nameList = geoName.getNames(altNames = True)
		for literal in self.literals_:
			if literal.getName() and literal.getName() in nameList:
				rv = True
				break
			
		return rv



	# legacy code
	#

	def applyForEach(self, f):
		for word in self.doc_.split(" "):
			f(self.doc_, word)
		

	def getContent(self):
		return self.doc_




	def deduplicate(self):
		self.doc_ = " ".join(set(self.doc_.split(" ")))
		return self


	def contains(self, word):
		rv = False
		wordStartPos = self.doc_.find(word)
		if (wordStartPos != -1):
			wordEndPos = (wordStartPos + len(word))
			if (wordEndPos < len(self.doc_)):
				if re.match('\\W', self.doc_[wordEndPos:(wordEndPos + 1)]):
					rv = True
		return rv
